﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace OuterParser.OsvitaComUa
{
    public class OsvitaComUaParserInterface
    {
      static   ParserOsvitaComUa parser = new ParserOsvitaComUa();
        public static void AddUnivs()
        {    
            OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog("osvita.com.ua start univ parsing");
            parser.GetDirectionInPage(@"http://www.osvita.com.ua/ua/universities/");
            parser.GetAllUnivHref();
            Console.WriteLine();
        }
        public static void AddProfession()
        {
            OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog("osvita.com.ua start profession parsing");
          //  parser.GetAllInfoAboutProfession(@"/ua/professions/hr-manager/","");
            parser.GetGroupOfProfessionHref(@"http://www.osvita.com.ua/ua/professions/", true);
            foreach (var profGroupHref in parser.groupOfProfessionHref.ToList())
            {
                parser.GetGroupOfProfessionHref(ParserOsvitaComUa.mainHref+ profGroupHref.Key, false);
            }
            foreach (var p in parser.ProfessionHref)
            {
                parser.GetAllInfoAboutProfession(p.Value, p.Key);
            }
        }
    }
    class ParserOsvitaComUa
    {
        static public string mainHref = @"http://www.osvita.com.ua";

        /// <summary>
        /// посилання на сторінки з університетами за напрямками підготовки
        /// </summary>
        public List<string> dirsHref = new List<string>();

        List<string> univHref = new List<string>();
        public List<KeyValuePair<string, string>> groupOfProfessionHref = new List<KeyValuePair<string, string>>();
        public List<KeyValuePair<string, string>> ProfessionHref = new List<KeyValuePair<string, string>>();
        int countOfUnivs = 0;
        public void GetDirectionInPage(string href)
        {
            int i = 0;
            do
            {
                try
                {
                    WebRequest firstRequest = WebRequest.Create(href);

                    using (var response = firstRequest.GetResponse())
                    {
                        using (var stream = response.GetResponseStream())
                        {
                            using (var reader = new StreamReader(stream, Encoding.GetEncoding(65001)))//utf-8
                            {
                                string allText = reader.ReadToEnd();
                                string forLength = "<h3>Пошук ВНЗ за напрямками</h3>";
                                allText = allText.Remove(0, allText.IndexOf(forLength) + forLength.Length);
                                allText = allText.Remove(allText.IndexOf("<h3>Пошук ВНЗ по містах</h3>"));
                                allText = allText.Remove(allText.IndexOf("<div"));
                                allText = allText.Replace("<ul class=\"direct2\" style=\"float:left\">", "").Replace("</ul>", "");

                                int index = 0;
                                string linkStr = "";
                                do
                                {
                                    index = allText.IndexOf("</a>");
                                    if (index == -1) break;
                                    linkStr = allText.Remove(index);
                                    allText = allText.Remove(0, index + 4);

                                    linkStr = linkStr.Remove(0, linkStr.IndexOf("\"") + 1);
                                    linkStr = linkStr.Remove(linkStr.IndexOf("\""));
                                    dirsHref.Add(linkStr);
                                }
                                while (true);
                                OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog("direction links add");
                                return;

                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);
                    OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog(string.Format("error parse direction href={0} \n{1}", href, e.ToString()));
                    if (i > 10) i++;
                    else break;
                }
            } while (true);

        }
        public void GetAllUnivHref()
        {
            foreach (string s in dirsHref)
            {
                GetHrefToUniversityInPage(mainHref + s);
            }
            OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog("add all univ href");

            countOfUnivs = 1;
            Console.WriteLine("univ add too db procces start");

            foreach (string st in univHref)
            {
                GetAllInfoAboutUniv(mainHref + st);
                Console.WriteLine(countOfUnivs++);
            }
        }
        public void GetHrefToUniversityInPage(string href)
        {
            int i = 0;
            string nextPage = "";
            do
            {
                try
                {
                    WebRequest firstRequest = WebRequest.Create(href);

                    using (var response = firstRequest.GetResponse())
                    {
                        using (var stream = response.GetResponseStream())
                        {
                            using (var reader = new StreamReader(stream, Encoding.GetEncoding(65001)))//utf-8
                            {
                                string allText = reader.ReadToEnd();
                                allText = allText.Remove(0, allText.IndexOf("paddContent"));
                                allText = allText.Remove(0, allText.IndexOf("</form>"));
                                allText = allText.Remove(0, allText.IndexOf("<tbody>") + 7);
                                allText = allText.Remove(0, allText.IndexOf("f_bor"));

                                //next page
                                string addstr = "наступна";
                                if (allText.Contains(addstr))
                                {
                                    nextPage = allText.Remove(allText.IndexOf(addstr));
                                    if (nextPage.Contains("попередня")) nextPage = nextPage.Remove(0, nextPage.IndexOf("попередня"));
                                    nextPage = nextPage.Remove(0, nextPage.LastIndexOf("<a") + 2);
                                    if (nextPage.Contains("href"))
                                    {
                                        nextPage = nextPage.Remove(0, nextPage.IndexOf("href"));
                                        nextPage = nextPage.Remove(0, nextPage.IndexOf("\"") + 1);
                                        nextPage = nextPage.Remove(nextPage.IndexOf("\""));
                                    }
                                    else
                                    {
                                        nextPage = "";
                                    }
                                }
                                allText = allText.Remove(allText.IndexOf("</table>"));

                                int index = 0;
                                do
                                {
                                    index = allText.IndexOf("</tr>");
                                    if (index == -1) break;
                                    addstr = allText.Remove(index);
                                    allText = allText.Remove(0, index + 5);

                                    addstr = addstr.Remove(0, addstr.IndexOf("href=\"") + 6);
                                    addstr = addstr.Remove(addstr.IndexOf("\""));
                                    if (!univHref.Contains(addstr))
                                    {
                                        univHref.Add(addstr);
                                        countOfUnivs++;
                                        Console.WriteLine(countOfUnivs);
                                    }
                                }
                                while (true);
                            }
                        }
                    }
                    if (nextPage != "")
                    {
                        GetHrefToUniversityInPage(mainHref + nextPage);
                    }
                    return;
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);
                    OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog(string.Format("error parse all university pages href={0} \n{1}", href, e.ToString()));
                    if (i > 10) i++;
                    else break;
                }
            } while (true);
        }
        public void GetAllInfoAboutUniv(string href)
        {
            string allText = GetHTMLByHref(href);
            //name
            string name = allText.Remove(0, allText.IndexOf("<h1 itemprop=\"name\""));
            name = name.Remove(0, name.IndexOf(">") + 1);
            name = name.Remove(name.IndexOf("<"));
            //short name ?????
            string nameShort = "";
            string[] nStr = name.Split(' ');
            if (name.Contains("("))
            {
                nameShort = name.Remove(name.IndexOf(")"));
                nameShort = nameShort.Remove(0, nameShort.IndexOf("(") + 1);
                name = name.Remove(name.IndexOf("("));
            }
            else
            {
                if (nStr[nStr.Length - 1].Length < 10)
                {
                    nameShort = nStr[nStr.Length - 1];
                    name = name.Replace(nameShort, "");
                }
            }

            Models.OsvitaComUa_University univ = new Models.OsvitaComUa_University();

            univ.Name = name;
            univ.ShortName = nameShort;
            univ.SourceHref = href;
            //table
            bool isTable = false;
            if (allText.Contains("class=\"tbl_grey_center nf\""))
            {
                string table = allText.Remove(0, allText.IndexOf("class=\"tbl_grey_center nf\""));
                if (table.Contains("Студентів"))
                {
                    table = table.Remove(0, table.IndexOf("Студентів"));
                    isTable = true;
                }
                else
                {
                    if (table.Contains("Викладачів"))
                    {
                        table = table.Remove(0, table.IndexOf("Викладачів"));
                        isTable = true;
                    }
                    else
                    {
                        if (table.Contains("Кандидатів наук"))
                        {
                            table = table.Remove(0, table.IndexOf("Кандидатів наук"));
                            isTable = true;
                        }
                        else
                        {
                            if (table.Contains("Професорів і докторів наук"))
                            {
                                table = table.Remove(0, table.IndexOf("Професорів і докторів наук"));
                                isTable = true;
                            }
                        }
                    }
                }
                if (isTable)
                {
                    table = table.Remove(table.IndexOf("</table>"));
                    string str = "";
                    string val = "";
                    int index = 0;
                    do
                    {
                        index = table.IndexOf("</tr>");
                        if (index == -1) break;
                        str = table.Remove(index);
                        table = table.Remove(0, index + 5);

                        val = str.Remove(str.LastIndexOf("</td>"));
                        val = val.Remove(0, val.LastIndexOf(">") + 1);
                        if (val != "")
                        {
                            if (str.Contains("Студентів")) univ.CountOfStydent = int.Parse(val);
                            if (str.Contains("Викладачів")) univ.CountOfTeacher = int.Parse(val);
                            if (str.Contains("Кандидатів наук")) univ.CountOfCandidatesOfSciences = int.Parse(val);
                            if (str.Contains("Професорів і докторів наук")) univ.CountOfProfessorwsAndDoctors = int.Parse(val);
                        }
                    }
                    while (true);
                }
            }
            //Abit info
            if (allText.Contains("<a name=\"entrant"))
            {
                univ.ForAbiturientsHTML = allText.Remove(0, allText.IndexOf("entrant") + 7);
                univ.ForAbiturientsHTML = allText.Remove(0, allText.IndexOf("<span>Абітурієнтам</span>"));
                univ.ForAbiturientsHTML = univ.ForAbiturientsHTML.Remove(0, univ.ForAbiturientsHTML.IndexOf("</div>") + 6);
                if (univ.ForAbiturientsHTML.Contains("<a name=\"about\""))
                {
                    univ.ForAbiturientsHTML = univ.ForAbiturientsHTML.Remove(univ.ForAbiturientsHTML.IndexOf("<a name=\"about\""));
                }
                else
                {
                    univ.ForAbiturientsHTML = univ.ForAbiturientsHTML.Remove(univ.ForAbiturientsHTML.IndexOf("<a"));
                }
            }
            //Detail info
            if (allText.Contains("<a name=\"about\""))
            {
                univ.DetailInfoAboutUniv = allText.Remove(0, allText.IndexOf("<a name=\"about\""));
                univ.DetailInfoAboutUniv = univ.DetailInfoAboutUniv.Remove(0, univ.DetailInfoAboutUniv.IndexOf("</div>") + 6);
                if (univ.DetailInfoAboutUniv.Contains("<a name=\"gallery\""))
                {
                    univ.DetailInfoAboutUniv = univ.DetailInfoAboutUniv.Remove(univ.DetailInfoAboutUniv.IndexOf("<a name=\"gallery\""));
                }
                else
                {
                    univ.DetailInfoAboutUniv = univ.DetailInfoAboutUniv.Remove(univ.DetailInfoAboutUniv.IndexOf("<a"));
                }
            }

            //add univ to dataBase
            OsvitaComUaServices.AddUniv(univ);

            //galery
            List<string> imageList = new List<string>();
            if (allText.Contains("class=\"gallery\">"))
            {
                string galery = allText.Remove(0, allText.IndexOf("class=\"gallery\">"));
                galery = galery.Remove(galery.IndexOf("<script"));

                int index = 0;
                string imgHref = "";
                do
                {
                    index = galery.IndexOf("</div>");
                    if (index == -1) break;
                    imgHref = galery.Remove(index);
                    if (!imgHref.Contains("class=\"pic")) break;
                    galery = galery.Remove(0, index + 6);

                    imgHref = imgHref.Remove(0, imgHref.IndexOf("src=\"") + 5);
                    imgHref = imgHref.Remove(imgHref.IndexOf("\""));
                    imageList.Add(imgHref);
                }
                while (true);

                //add image
                int id = OsvitaComUaServices.GetUnivIdByName(univ.Name);
                if (id != -1)
                {
                    for (int i = 0; i < imageList.Count; i++)
                    {
                        OsvitaComUaServices.AddImage(imageList[i], i.ToString(), id);
                    }

                    //logo
                    if (allText.Contains("gerb"))
                    {
                        string logo = allText.Remove(0, allText.IndexOf("gerb") + 4);
                        logo = logo.Remove(logo.IndexOf("</div>"));
                        if (logo.Contains("src"))
                        {
                            logo = logo.Remove(0, logo.IndexOf("src=\"") + 5);
                            logo = logo.Remove(logo.IndexOf("\""));
                            OsvitaComUaServices.AddImage(logo, "logo", id);
                        }
                    }
                }
            }


        }
        public string GetHTMLByHref(string href)
        {
            int i = 0;
            do
            {
                try
                {
                    WebRequest firstRequest = WebRequest.Create(href);

                    using (var response = firstRequest.GetResponse())
                    {
                        using (var stream = response.GetResponseStream())
                        {
                            using (var reader = new StreamReader(stream, Encoding.GetEncoding(65001)))//utf-8
                            {
                                return reader.ReadToEnd();
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);
                    OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog(string.Format("error parse direction href={0} \n{1}", href, e.ToString()));
                    if (i > 10) i++;
                    else break;
                }
            } while (true);
            return null;
        }
        string Get2UlDirectIdTo1Li(string allText, string href)
        {
            string ulLeft = "";
            if (allText.Contains("direct1"))
            {
               ulLeft= allText.Remove(0, allText.IndexOf("direct1"));
                ulLeft = ulLeft.Remove(0, ulLeft.IndexOf(">") + 1);
                ulLeft = ulLeft.Remove(ulLeft.IndexOf("</ul>"));
            }
            string ulRith = "";
            if (allText.Contains("direct2"))
            {
               ulRith= allText.Remove(0, allText.IndexOf("direct2"));
                ulRith = ulRith.Remove(0, ulRith.IndexOf(">") + 1);
                ulRith = ulRith.Remove(ulRith.IndexOf("</ul>"));
            }
            string allLi = ulLeft + ulRith;
            return allLi;
        }
        public void GetGroupOfProfessionHref(string href,bool isGroupOfProfession)
        {
            string allText = GetHTMLByHref( href);
            if (allText != null)
            {
                string allLi = Get2UlDirectIdTo1Li( allText, href);
           
            string name = "";
            string link = "";
            int index = 0;
            do
            {
                index = allLi.IndexOf("</li>");
                if (index == -1) break;
                name = allLi.Remove(index);
                allLi = allLi.Remove(0, index + 5);

                name = name.Remove(name.IndexOf("</a>"));
                link = name.Remove(name.LastIndexOf("\""));
                link = link.Remove(0,link.IndexOf("\"")+1);
                name = name.Remove(0, name.LastIndexOf(">")+1);
                if (isGroupOfProfession)
                {
                    groupOfProfessionHref.Add(new KeyValuePair<string,string>(link, name));
                    OsvitaComUaServices.AddGroupOfProfession(new Models.OsvitaComUa_GroupOfProfession() { Name = name });
                }
                else
                {
                    ProfessionHref.Add(new KeyValuePair<string,string>( groupOfProfessionHref.FirstOrDefault(x=>x.Key==href.Replace(mainHref,"")).Value,link));
                }
            }
            while (true);
            }
            else
            {
                Console.WriteLine();
            }

        }

        public void GetAllInfoAboutProfession(string href, string groupName)
        {
            string allText = GetHTMLByHref(mainHref+href);
            Models.OsvitaComUa_Profession profession = new Models.OsvitaComUa_Profession();
            profession.GroupOfProfessionId = OsvitaComUaServices.GetGroupOfProfessionIdByName(groupName);
            if (profession.GroupOfProfessionId == -1) return;
            allText = allText.Remove(0,allText.IndexOf("id=\"paddContent\""));
            allText = allText.Remove(allText.IndexOf("div id=\"work-informer32\""));
            allText=allText.Remove(0,allText.IndexOf("</style>")+8);
            allText=allText.Remove(allText.IndexOf("<link"));
            //all text
            profession.AllText=allText;
            //source href
            profession.SourceHref=mainHref+href;
            //name
            profession.Name = allText.Remove(0, allText.IndexOf("<h1>") + 4);
            profession.Name = profession.Name.Remove(profession.Name.IndexOf("</h1>"));
            
            //specific
            if (allText.Contains("Специфіка професії</h3>"))
            {
                profession.TheSpecificityOfTheProfession = allText.Remove(0, allText.IndexOf("Специфіка професії</h3>")+23);
                if (profession.TheSpecificityOfTheProfession.Contains("<h3")) profession.TheSpecificityOfTheProfession = profession.TheSpecificityOfTheProfession.Remove(profession.TheSpecificityOfTheProfession.IndexOf("<h3"));
            }

            //place of work
            if (allText.Contains("Місце роботи</h3>"))
            {
                profession.PlaceOfJob = allText.Remove(0, allText.IndexOf("Місце роботи</h3>")+17);
                if (profession.PlaceOfJob.Contains("<h3")) profession.PlaceOfJob = profession.PlaceOfJob.Remove(profession.PlaceOfJob.IndexOf("<h3"));
            }

            //personal characteristics
            if (allText.Contains("Особисті якості</h3>"))
            {
                profession.PersonalCharacteristics = allText.Remove(0, allText.IndexOf("Особисті якості</h3>")+20);
                if (profession.PersonalCharacteristics.Contains("<h3")) profession.PersonalCharacteristics = profession.PersonalCharacteristics.Remove(profession.PersonalCharacteristics.IndexOf("<h3"));
            }

            //career
            if (allText.Contains("Кар'єра й зарплата</h3>"))
            {
                profession.Career = allText.Remove(0, allText.IndexOf("Кар'єра й зарплата</h3>")+23);
                if (profession.Career.Contains("<h3")) profession.Career = profession.Career.Remove(profession.Career.IndexOf("<h3"));
            }

            OsvitaComUaServices.AddProfession(profession);
            
        }

    }
    class OsvitaComUaServices
    {
        static string imageFolder = @"D:\pulsatrix\pulsatrix\source\OsvitaComUaSource\";
        static Models.PulsStagingDbEntities db = new Models.PulsStagingDbEntities();

        public static int GetUnivIdByName(string name)
        {
            var a = db.OsvitaComUa_University.FirstOrDefault(x => x.Name == name);
            if (a == null) return -1;
            return a.Id;
        }
        public static int GetGroupOfProfessionIdByName(string name)
        {
            var a = db.OsvitaComUa_GroupOfProfession.FirstOrDefault(x => x.Name == name);
            if (a == null) return -1;
            return a.Id;
        }
        public static int GetProfessionIdByName(string name)
        {
            var a = db.OsvitaComUa_Profession.FirstOrDefault(x => x.Name == name);
            if (a == null) return -1;
            return a.Id;
        }
        static public void AddUniv(Models.OsvitaComUa_University univ)
        {
            try
            {
                if (GetUnivIdByName(univ.Name) != -1) return;
                db.OsvitaComUa_University.Add(univ);
                db.SaveChanges();
            }
            catch (Exception e)
            {
                OuterParser.OsvitaUa.Notification.AddErrorToErrorsLog(string.Format("univ add error \n{0}", e.ToString()));
            }
        }
        static public void AddGroupOfProfession(Models.OsvitaComUa_GroupOfProfession group)
        {
            if (GetGroupOfProfessionIdByName(group.Name) != -1) return;
            db.OsvitaComUa_GroupOfProfession.Add(group);
            db.SaveChanges();
        }
        static public void AddProfession(Models.OsvitaComUa_Profession prof)
        {
            if (GetProfessionIdByName(prof.Name) != -1) return;
            db.OsvitaComUa_Profession.Add(prof);
            db.SaveChanges();
        }
        public static int GetImageIdByName(string nameImg, int univId)
        {
            var a = db.OsvitaComUa_Images.FirstOrDefault(x => x.ImageName == nameImg && x.UniversityId == univId);
            if (a == null) return -1;
            return a.Id;
        }
        static public void AddImageIdToDb(string imageName, int univId)
        {
            if (GetImageIdByName(imageName, univId) != -1) return;
            db.OsvitaComUa_Images.Add(new Models.OsvitaComUa_Images() { UniversityId = univId, ImageName = imageName });
            db.SaveChanges();
        }
        static public void AddImage(string href, string addName, int univId)
        {
            try
            {
                WebClient wc = new WebClient();
                string fullName = univId.ToString() + "_" + addName + ".jpg";
                href = ParserOsvitaComUa.mainHref + href;
                if (System.IO.File.Exists(imageFolder + fullName)) return;
                wc.DownloadFile(href, imageFolder + fullName);
                AddImageIdToDb(fullName, univId);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.ToString());
            }
        }

    }
}
